{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Copyright 2015 The TensorFlow Authors. All Rights Reserved.\n",
    "#\n",
    "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "# you may not use this file except in compliance with the License.\n",
    "# You may obtain a copy of the License at\n",
    "#\n",
    "#     http://www.apache.org/licenses/LICENSE-2.0\n",
    "#\n",
    "# Unless required by applicable law or agreed to in writing, software\n",
    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "# See the License for the specific language governing permissions and\n",
    "# limitations under the License.\n",
    "# ==============================================================================\n",
    "\"\"\"Basic word2vec example.\"\"\"\n",
    "\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import collections\n",
    "import math\n",
    "import os\n",
    "import random\n",
    "from tempfile import gettempdir\n",
    "import zipfile\n",
    "\n",
    "import numpy as np\n",
    "from six.moves import urllib\n",
    "from six.moves import xrange  # pylint: disable=redefined-builtin\n",
    "import tensorflow as tf\n",
    "import codecs \n",
    "import json\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data size 1786003\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Step 1: read data \n",
    "# Read the data into a list of strings.\n",
    "def read_data(filename):\n",
    "  with codecs.open(filename,encoding='utf-8') as f:\n",
    "    data = list(f.read().replace('\\r','').replace('\\n',''))\n",
    "  return data\n",
    "\n",
    "vocabulary = read_data('QuanSongCi.txt')\n",
    "print('Data size', len(vocabulary))\n",
    "# print(vocabulary)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most common words (+UNK) [['UNK', 1194], ('。', 149620), ('，', 108451), ('、', 19612), ('人', 13607)]\n",
      "Sample data [1509, 1830, 39, 612, 46, 9, 111, 116, 7, 8] ['潘', '阆', '酒', '泉', '子', '（', '十', '之', '一', '）']\n"
     ]
    }
   ],
   "source": [
    "# Step 2: Build the dictionary and replace rare words with UNK token.\n",
    "vocabulary_size = 5000\n",
    "\n",
    "\n",
    "def build_dataset(words, n_words):\n",
    "  \"\"\"Process raw inputs into a dataset.\"\"\"\n",
    "  count = [['UNK', -1]]\n",
    "  count.extend(collections.Counter(words).most_common(n_words - 1))\n",
    "  dictionary = dict()\n",
    "  for word, _ in count:\n",
    "    dictionary[word] = len(dictionary)\n",
    "  data = list()\n",
    "  unk_count = 0\n",
    "  for word in words:\n",
    "    index = dictionary.get(word, 0)\n",
    "    if index == 0:  # dictionary['UNK']\n",
    "      unk_count += 1\n",
    "    data.append(index)\n",
    "  count[0][1] = unk_count\n",
    "  reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "  return data, count, dictionary, reversed_dictionary\n",
    "\n",
    "# Filling 4 global variables:\n",
    "# data - list of codes (integers from 0 to vocabulary_size-1).\n",
    "#   This is the original text but words are replaced by their codes\n",
    "# count - map of words(strings) to count of occurrences\n",
    "# dictionary - map of words(strings) to their codes(integers)\n",
    "# reverse_dictionary - maps codes(integers) to words(strings)\n",
    "data, count, dictionary, reverse_dictionary = build_dataset(vocabulary,\n",
    "                                                            vocabulary_size)\n",
    "del vocabulary  # Hint to reduce memory.\n",
    "print('Most common words (+UNK)', count[:5])\n",
    "print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]])\n",
    "with codecs.open('dictionary.json','w',encoding='utf-8') as outfile:  \n",
    "    json.dump(dictionary,outfile,ensure_ascii=False)  \n",
    "with codecs.open('reverse_dictionary.json','w',encoding='utf-8') as outfile:  \n",
    "    json.dump(reverse_dictionary,outfile,ensure_ascii=False)  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1830 阆 -> 1509 潘\n",
      "1830 阆 -> 39 酒\n",
      "39 酒 -> 1830 阆\n",
      "39 酒 -> 612 泉\n",
      "612 泉 -> 39 酒\n",
      "612 泉 -> 46 子\n",
      "46 子 -> 612 泉\n",
      "46 子 -> 9 （\n"
     ]
    }
   ],
   "source": [
    "data_index = 0\n",
    "\n",
    "# Step 3: Function to generate a training batch for the skip-gram model.\n",
    "def generate_batch(batch_size, num_skips, skip_window):\n",
    "  global data_index\n",
    "  assert batch_size % num_skips == 0\n",
    "  assert num_skips <= 2 * skip_window\n",
    "  batch = np.ndarray(shape=(batch_size), dtype=np.int32)\n",
    "  labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)\n",
    "  span = 2 * skip_window + 1  # [ skip_window target skip_window ]\n",
    "  buffer = collections.deque(maxlen=span)\n",
    "  if data_index + span > len(data):\n",
    "    data_index = 0\n",
    "  buffer.extend(data[data_index:data_index + span])\n",
    "  data_index += span\n",
    "  for i in range(batch_size // num_skips):\n",
    "    context_words = [w for w in range(span) if w != skip_window]\n",
    "    words_to_use = random.sample(context_words, num_skips)\n",
    "    for j, context_word in enumerate(words_to_use):\n",
    "      batch[i * num_skips + j] = buffer[skip_window]\n",
    "      labels[i * num_skips + j, 0] = buffer[context_word]\n",
    "    if data_index == len(data):\n",
    "      buffer.extend(data[:span])\n",
    "      data_index = span\n",
    "    else:\n",
    "      buffer.append(data[data_index])\n",
    "      data_index += 1\n",
    "  # Backtrack a little bit to avoid skipping words in the end of a batch\n",
    "  data_index = (data_index + len(data) - span) % len(data)\n",
    "  return batch, labels\n",
    "\n",
    "batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1)\n",
    "for i in range(8):\n",
    "  print(batch[i], reverse_dictionary[batch[i]],\n",
    "        '->', labels[i, 0], reverse_dictionary[labels[i, 0]])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Step 4: Build and train a skip-gram model.\n",
    "\n",
    "batch_size = 128\n",
    "embedding_size = 128  # Dimension of the embedding vector.\n",
    "skip_window = 1       # How many words to consider left and right.\n",
    "num_skips = 2         # How many times to reuse an input to generate a label.\n",
    "num_sampled = 64      # Number of negative examples to sample.\n",
    "\n",
    "# We pick a random validation set to sample nearest neighbors. Here we limit the\n",
    "# validation samples to the words that have a low numeric ID, which by\n",
    "# construction are also the most frequent. These 3 variables are used only for\n",
    "# displaying model accuracy, they don't affect calculation.\n",
    "valid_size = 16     # Random set of words to evaluate similarity on.\n",
    "valid_window = 100  # Only pick dev samples in the head of the distribution.\n",
    "valid_examples = np.random.choice(valid_window, valid_size, replace=False)\n",
    "\n",
    "\n",
    "graph = tf.Graph()\n",
    "\n",
    "with graph.as_default():\n",
    "\n",
    "  # Input data.\n",
    "  train_inputs = tf.placeholder(tf.int32, shape=[batch_size])\n",
    "  train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])\n",
    "  valid_dataset = tf.constant(valid_examples, dtype=tf.int32)\n",
    "\n",
    "  # Ops and variables pinned to the CPU because of missing GPU implementation\n",
    "  with tf.device('/cpu:0'):\n",
    "    # Look up embeddings for inputs.\n",
    "    embeddings = tf.Variable(\n",
    "        tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))\n",
    "    embed = tf.nn.embedding_lookup(embeddings, train_inputs)\n",
    "\n",
    "    # Construct the variables for the NCE loss\n",
    "    nce_weights = tf.Variable(\n",
    "        tf.truncated_normal([vocabulary_size, embedding_size],\n",
    "                            stddev=1.0 / math.sqrt(embedding_size)))\n",
    "    nce_biases = tf.Variable(tf.zeros([vocabulary_size]))\n",
    "\n",
    "  # Compute the average NCE loss for the batch.\n",
    "  # tf.nce_loss automatically draws a new sample of the negative labels each\n",
    "  # time we evaluate the loss.\n",
    "  # Explanation of the meaning of NCE loss:\n",
    "  #   http://mccormickml.com/2016/04/19/word2vec-tutorial-the-skip-gram-model/\n",
    "  loss = tf.reduce_mean(\n",
    "      tf.nn.nce_loss(weights=nce_weights,\n",
    "                     biases=nce_biases,\n",
    "                     labels=train_labels,\n",
    "                     inputs=embed,\n",
    "                     num_sampled=num_sampled,\n",
    "                     num_classes=vocabulary_size))\n",
    "\n",
    "  # Construct the SGD optimizer using a learning rate of 1.0.\n",
    "  optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)\n",
    "\n",
    "  # Compute the cosine similarity between minibatch examples and all embeddings.\n",
    "  norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))\n",
    "  normalized_embeddings = embeddings / norm\n",
    "  valid_embeddings = tf.nn.embedding_lookup(\n",
    "      normalized_embeddings, valid_dataset)\n",
    "  similarity = tf.matmul(\n",
    "      valid_embeddings, normalized_embeddings, transpose_b=True)\n",
    "\n",
    "  # Add variable initializer.\n",
    "  init = tf.global_variables_initializer()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Average loss at step  0 :  195.577575684\n",
      "Nearest to 是: 舅, 仃, 航, 党, 睨, 挝, 工, 回,\n",
      "Nearest to 三: 维, 有, 滂, 轰, 崔, 经, 渭, 棚,\n",
      "Nearest to 行: 贾, 蒂, 赌, 飐, 且, 岊, 迄, 壕,\n",
      "Nearest to 中: 裸, 艘, 阍, 臧, 笋, 小, 惶, 珑,\n",
      "Nearest to 生: 绳, 缵, 期, 池, 邻, 丐, 史, 茹,\n",
      "Nearest to 东: 等, 着, 论, 阴, 娘, 忤, 穿, 桃,\n",
      "Nearest to 似: 门, 碌, 萄, 惔, 铺, 醒, 噤, 龌,\n",
      "Nearest to 多: 儿, 控, 缬, 挽, 巴, 镊, , 罔,\n",
      "Nearest to 前: 旟, 阍, 憀, 觉, 羹, 寓, 琥, 潦,\n",
      "Nearest to 深: 埒, 盲, 关, 陇, 鹚, 掀, 五, 7,\n",
      "Nearest to 重: 揖, 娑, 貔, 授, 蓂, 核, 资, 助,\n",
      "Nearest to 更: 贾, 艳, 禊, 板, 嚎, 办, 澳, 骈,\n",
      "Nearest to 君: 醿, 颖, 渐, 讨, 孽, 辍, 选, 半,\n",
      "Nearest to ，: 簦, 庞, 布, 辑, 浦, 桷, 赁, 解,\n",
      "Nearest to 处: 撄, 灏, 烛, 谟, 刀, 药, 樾, 藕,\n",
      "Nearest to 寒: 忡, 伦, 鞞, 株, 刀, 阒, 辔, 伪,\n",
      "Average loss at step  2000 :  21.9127893109\n",
      "Average loss at step  4000 :  5.33549789762\n",
      "Average loss at step  6000 :  4.97939793253\n",
      "Average loss at step  8000 :  4.77627922201\n",
      "Average loss at step  10000 :  4.65908667749\n",
      "Nearest to 是: 舅, 工, 睨, 准, 欢,  , 航, 兼,\n",
      "Nearest to 三: 维, 扁, 渭, 崔, 畦, 经, 十, 饰,\n",
      "Nearest to 行: 贾, 且, 蒂, 厢, 飐, 雨, 效, 别,\n",
      "Nearest to 中: 小,  , 纵, 珑, 笋, 几, 瀼, 肢,\n",
      "Nearest to 生: 筠,  , 绳, 摧, 池, 骑, 认, 哉,\n",
      "Nearest to 东: 着, 等, 阴, 废, 穿, 注, 娘, 论,\n",
      "Nearest to 似: 门, 碌, 铺, 龌, 醒, 窕, 脑, 损,\n",
      "Nearest to 多: 儿, 巴, 控, 缬, 镊, 扆, 闭, 挽,\n",
      "Nearest to 前: 旟, 叠, 寓, 羹, 觉, 湍, 琥, 褐,\n",
      "Nearest to 深: 巧, 关, 掀, 步, 萨, 魏, 表, 陇,\n",
      "Nearest to 重: 揖, 授, 蓂, 垂, 资, 核, 岁, 娑,\n",
      "Nearest to 更: 贾, 禊, 板, 艳, 惨, 恨, 嗅, 殢,\n",
      "Nearest to 君: 醿, 选, 丁, 俱, 渐, 潭, 藕, 道,\n",
      "Nearest to ，: 。, 、,  , 即, ）, 没, 牒, 俱,\n",
      "Nearest to 处: 藕, 刀, 灏, 系, 探, 皓, 莘, 徉,\n",
      "Nearest to 寒: 鞞, 稀, 伦, 忡, 岭, 株, 枨, 迹,\n",
      "Average loss at step  12000 :  4.60754791796\n",
      "Average loss at step  14000 :  4.57150185919\n",
      "Average loss at step  16000 :  4.56309955132\n",
      "Average loss at step  18000 :  4.57166467714\n",
      "Average loss at step  20000 :  4.49895702851\n",
      "Nearest to 是: 舅, 睨, 准, 航, 来, 党,  , 忄,\n",
      "Nearest to 三: 维, 扁, 渭, 经, 十, 畦, 有, 剡,\n",
      "Nearest to 行: 贾, 厢, 且, 雨, 蒂, 别, 飐, 效,\n",
      "Nearest to 中: 珑,  , 小, 瀼, 几, 纵, 答, 阍,\n",
      "Nearest to 生: 丐, 筠, 摧,  , 绳, 认, 从, A,\n",
      "Nearest to 东: 着, 等, 废, 圻, 注, 峭, 北, 坤,\n",
      "Nearest to 似: 碌, 门, 在, 如, 铺, 龌, 脑, 论,\n",
      "Nearest to 多: 巴, 缬, 控, 儿, 镊, 扆, 也, 闭,\n",
      "Nearest to 前: 旟, 湍, 寓, 阍, 褐, 叠, 羹, 琥,\n",
      "Nearest to 深: 巧, 关, 掀, 魏, 萨, 齁, 步, 茨,\n",
      "Nearest to 重: 授, 蓂, 揖, 搀, 资, 垂, 钺, 搔,\n",
      "Nearest to 更: 贾, 板, 禊, 艳, 惨, 恨, 嗅, 徐,\n",
      "Nearest to 君: 选, 醿, 我, 道, 赊, 谁, 懵, 坤,\n",
      "Nearest to ，: 、, 。,  , ）, 即, 握, 没, 牒,\n",
      "Nearest to 处: 系, 莘, 藕, 榕, 人, 拼, 灏, 探,\n",
      "Nearest to 寒: 鞞, 枨, 稀, 伦, 秋, 覆, 迹, 伪,\n",
      "Average loss at step  22000 :  4.44426261508\n",
      "Average loss at step  24000 :  4.47614934099\n",
      "Average loss at step  26000 :  4.52709369361\n",
      "Average loss at step  28000 :  4.45032344329\n",
      "Average loss at step  30000 :  4.25379583466\n",
      "Nearest to 是: 睨, 来, 舅,  , 在, 党, 有, 航,\n",
      "Nearest to 三: 维, 扁, 十, 渭, 经, 畦, 剡, 之,\n",
      "Nearest to 行: 贾, 厢, 且, 暨, 蒂, 效, 飏, 雨,\n",
      "Nearest to 中: 瀼, 答, 裸, 珑, 臧,  , 几, 柝,\n",
      "Nearest to 生: 缵, 丐, 筠, 摧,  , 魏, 予, 骑,\n",
      "Nearest to 东: 着, 废, 西, 北, 等, 圻, 注, 坤,\n",
      "Nearest to 似: 如, 碌, 在, 龌, 门, 脑, 铺, 论,\n",
      "Nearest to 多: 缬, 巴, 控, 镊, 也, 儿, 却, 扆,\n",
      "Nearest to 前: 旟, 湍, 阍, 褐, 寓, 缟, 叠, 赊,\n",
      "Nearest to 深: 关, 巧, 茨, 掀, 萨, 魏, 齁, 放,\n",
      "Nearest to 重: 授, 囗, 揖, 搀, 蓂, 傲, 垂, 漆,\n",
      "Nearest to 更: 贾, 板, 禊, 恨, 嗅, 惨, 徐, 艳,\n",
      "Nearest to 君: 我, 谁, 选, 囗, 醿, 返, 懵, 道,\n",
      "Nearest to ，: 、, 。,  , ）, 即, 握, 牒, 俱,\n",
      "Nearest to 处: 也, 系, 莘, 人, 榕, 藕, 拼, 灏,\n",
      "Nearest to 寒: 秋, 枨, 鞞, 伦, 稀, 晴, 覆, 暖,\n",
      "Average loss at step  32000 :  4.29143829739\n",
      "Average loss at step  34000 :  4.30550097239\n",
      "Average loss at step  36000 :  4.27809699869\n",
      "Average loss at step  38000 :  4.26994148672\n",
      "Average loss at step  40000 :  4.28946552289\n",
      "Nearest to 是: 在, 有, 睨, 来, 舅,  , 党, 航,\n",
      "Nearest to 三: 维, 扁, 经, 十, 渭, 畦, 棚, 剡,\n",
      "Nearest to 行: 贾, 且, 厢, 暨, 别, 蒂, 效, 飏,\n",
      "Nearest to 中: 瀼, 答, 里, 茯, 柝, 臧,  , 乇,\n",
      "Nearest to 生: 缵, 丐, 筠, 怯,  , 摧, 予, 冬,\n",
      "Nearest to 东: 西, 废, 着, 北, 圻, 等, 注, 晤,\n",
      "Nearest to 似: 如, 在, 碌, 论, 铺, 囗, 龌, 踪,\n",
      "Nearest to 多: 巴, 缬, 也, 镊, 控, 扆, 却, 儿,\n",
      "Nearest to 前: 湍, 旟, 褐, 阍, 寓, 缟, 赊, 俏,\n",
      "Nearest to 深: 关, 巧, 茨, 萨, 魏, 齁, 掀, 著,\n",
      "Nearest to 重: 授, 搀, 囗, 蓂, 垂, 揖, 漆, 岁,\n",
      "Nearest to 更: 贾, 板, 禊, 但, 嗅, 恨, 螭, 始,\n",
      "Nearest to 君: 我, 谁, 囗, 选, 醿, 赊, 懵, 要,\n",
      "Nearest to ，: 、, 。, ）, 即,  , 俱, 盍, 郢,\n",
      "Nearest to 处: 榕, 也, 莘, 系, 灏, 借, 拼, 人,\n",
      "Nearest to 寒: 秋, 枨, 鞞, 暖, 晴, 覆, 稀, 伦,\n",
      "Average loss at step  42000 :  4.2943252548\n",
      "Average loss at step  44000 :  4.34119455671\n",
      "Average loss at step  46000 :  4.32611962926\n",
      "Average loss at step  48000 :  4.33150986171\n",
      "Average loss at step  50000 :  4.29032135475\n",
      "Nearest to 是: 在, 有, 睨, 党, 来, 舅, 到,  ,\n",
      "Nearest to 三: 维, 扁, 经, 十, 畦, 几, 百, 渭,\n",
      "Nearest to 行: 贾, 厢, 暨, 飏, 且, 别, 效, 纪,\n",
      "Nearest to 中: 里, 瀼, 答, 柝, 茯, 臧, 乇, 裸,\n",
      "Nearest to 生: 缵, 筠, 丐, 怯, 冬, 摧, 予, 认,\n",
      "Nearest to 东: 西, 废, 北, 着, 圻, 注, 南, 等,\n",
      "Nearest to 似: 如, 在, 碌, 、, 铺, 论, 猱, 龌,\n",
      "Nearest to 多: 巴, 缬, 镊, 也, 扆, 却, 奈, 甚,\n",
      "Nearest to 前: 湍, 旟, 褐, 阍, 缟, 俏, 晏, 赊,\n",
      "Nearest to 深: 关, 巧, 茨, 放, 齁, 著, 枨, 孰,\n",
      "Nearest to 重: 授, 囗, 漆, 搀, 猛, 蓂, 揖, 垂,\n",
      "Nearest to 更: 但, 恨, 始, 螭, 板, 陛, 禊, 贾,\n",
      "Nearest to 君: 我, 谁, 选, 囗, 懵, 要, 醿, 赊,\n",
      "Nearest to ，: 、, 。, ）,  , 即, 豉, 牒, 俱,\n",
      "Nearest to 处: 榕, 也, 灏, 莘, 拼, 系, 借, 醿,\n",
      "Nearest to 寒: 秋, 鞞, 伪, 暖, 枨, 伦, 晴, 覆,\n",
      "Average loss at step  52000 :  4.32241110492\n",
      "Average loss at step  54000 :  4.36158951378\n",
      "Average loss at step  56000 :  4.31867869079\n",
      "Average loss at step  58000 :  4.16917260861\n",
      "Average loss at step  60000 :  4.19165993369\n",
      "Nearest to 是: 在, 有, 来, 睨, 到, 党, 舅, 果,\n",
      "Nearest to 三: 维, 扁, 十, 二, 经, 七, 五, 畦,\n",
      "Nearest to 行: 贾, 暨, 效, 厢, 岊, 飏, 兮, 蒂,\n",
      "Nearest to 中: 里, 瀼, 答, 柝, 茯, 洁, 臧, 清,\n",
      "Nearest to 生: 缵, 怯, 筠, 丐, 摧, 魏, 损, 予,\n",
      "Nearest to 东: 西, 废, 北, 着, 南, 圻, 蘅, 注,\n",
      "Nearest to 似: 如, 在, 碌, 见, 、, 论, 铺, 踪,\n",
      "Nearest to 多: 巴, 缬, 也, 镊, 甚, 却, 扆, 奈,\n",
      "Nearest to 前: 旟, 湍, 阍, 褐, 玳, 俏, 缟, 晏,\n",
      "Nearest to 深: 关, 巧, 茨, 何, 静, 著, 浅, 放,\n",
      "Nearest to 重: 授, 漆, 搀, 垂, 囗, 猛, 蓂, 蕾,\n",
      "Nearest to 更: 但, 陛, 始, 又, 嗅, 莫, 禊, 窊,\n",
      "Nearest to 君: 我, 谁, 囗, 要, 伊, 选, 懵, 返,\n",
      "Nearest to ，: 、, 。, ）, 即,  , 俱, 港, 牒,\n",
      "Nearest to 处: 也, 榕, 莘, 意, 灏, 时, 拼, 谪,\n",
      "Nearest to 寒: 秋, 暖, 枨, 鞞, 晴, 伦, 覆, 暮,\n",
      "Average loss at step  62000 :  4.20402489328\n",
      "Average loss at step  64000 :  4.20325037694\n",
      "Average loss at step  66000 :  4.19100030959\n",
      "Average loss at step  68000 :  4.20379883277\n",
      "Average loss at step  70000 :  4.22214483702\n",
      "Nearest to 是: 在, 有, 到, 果, 睨, 道, 非, 党,\n",
      "Nearest to 三: 维, 经, 扁, 十, 二, 七, 畦, 六,\n",
      "Nearest to 行: 贾, 暨, 效, 厢, 飏, 岊, 别, 兮,\n",
      "Nearest to 中: 里, 瀼, 答, 柝, 臧, 洁, 乾, 茯,\n",
      "Nearest to 生: 怯, 缵, 丐, 筠, 予, 冬, 魏, 乙,\n",
      "Nearest to 东: 西, 北, 废, 圻, 南, 着, 蘅, 注,\n",
      "Nearest to 似: 如, 在, 碌, 论, 铺, 崦, 踪, 见,\n",
      "Nearest to 多: 巴, 甚, 也, 缬, 镊, 扆, 奈, 却,\n",
      "Nearest to 前: 旟, 湍, 阍, 俏, 褐, 玳, 晏, 缟,\n",
      "Nearest to 深: 关, 巧, 茨, 浅, 静, 著, 浓, 放,\n",
      "Nearest to 重: 授, 搀, 垂, 漆, 囗, 蕾, 蓂, 猛,\n",
      "Nearest to 更: 但, 陛, 始, 又, 窊, 且, 却, 螭,\n",
      "Nearest to 君: 我, 谁, 伊, 要, 囗, 选, 公, 懵,\n",
      "Nearest to ，: 、, 。, ）,  , 即, 俱, ,, 佣,\n",
      "Nearest to 处: 榕, 也, 时, 莘, 灏, 醿, 谪, 意,\n",
      "Nearest to 寒: 秋, 暖, 枨, 鞞, 覆, 暮, 晴, 伪,\n",
      "Average loss at step  72000 :  4.2585595957\n",
      "Average loss at step  74000 :  4.25655788553\n",
      "Average loss at step  76000 :  4.25859546256\n",
      "Average loss at step  78000 :  4.23225291431\n",
      "Average loss at step  80000 :  4.25443656886\n",
      "Nearest to 是: 在, 有, 到, 道, 果, 睨, 似, 杆,\n",
      "Nearest to 三: 维, 七, 二, 经, 扁, 十, 六, 五,\n",
      "Nearest to 行: 贾, 暨, 效, 厢, 飏, 别, 岊, 纪,\n",
      "Nearest to 中: 里, 瀼, 柝, 答, 臧, 茯, 清, 洁,\n",
      "Nearest to 生: 怯, 缵, 筠, 丐, 乙, 魏, 予, 冬,\n",
      "Nearest to 东: 西, 北, 废, 南, 圻, 蘅, 着, 逝,\n",
      "Nearest to 似: 如, 在, 碌, 见, 崦, 论, 猱, 铺,\n",
      "Nearest to 多: 巴, 也, 镊, 甚, 缬, 扆, 却, 浓,\n",
      "Nearest to 前: 旟, 湍, 阍, 晏, 玳, 俏, 褐, 苹,\n",
      "Nearest to 深: 关, 巧, 浅, 静, 茨, 怨, 孰, 著,\n",
      "Nearest to 重: 授, 漆, 蕾, 搀, 蓂, 垂, 猛, 伯,\n",
      "Nearest to 更: 又, 陛, 但, 始, 窊, 且, 螭, 嗅,\n",
      "Nearest to 君: 我, 谁, 伊, 要, 选, 公, 囗, 懵,\n",
      "Nearest to ，: 、, 。, ）, 俱, 佣, 牒, （, 即,\n",
      "Nearest to 处: 榕, 也, 莘, 时, 灏, 迥, 著, 谪,\n",
      "Nearest to 寒: 秋, 暖, 伪, 枨, 鞞, 暮, 伦, 晴,\n",
      "Average loss at step  82000 :  4.28907582581\n",
      "Average loss at step  84000 :  4.23616454363\n",
      "Average loss at step  86000 :  4.13020865786\n",
      "Average loss at step  88000 :  4.14252015865\n",
      "Average loss at step  90000 :  4.1436214757\n",
      "Nearest to 是: 在, 有, 到, 似, 果, 道, 把, 杆,\n",
      "Nearest to 三: 二, 五, 七, 六, 维, 十, 经, 扁,\n",
      "Nearest to 行: 贾, 效, 暨, 厢, 岊, 送, 飏, 贱,\n",
      "Nearest to 中: 里, 瀼, 柝, 答, 茯, 清, 臧, 洁,\n",
      "Nearest to 生: 怯, 缵, 筠, 丐, 乙, 魏, 予, 冬,\n",
      "Nearest to 东: 西, 北, 废, 南, 圻, 蘅, 逝, 咸,\n",
      "Nearest to 似: 如, 在, 见, 是, 碌, 崦, 铺, 记,\n",
      "Nearest to 多: 甚, 镊, 巴, 也, 缬, 浓, 却, 浑,\n",
      "Nearest to 前: 旟, 玳, 俏, 湍, 阍, 晏, 褐, 下,\n",
      "Nearest to 深: 关, 巧, 浅, 静, 何, 茨, 怨, 浓,\n",
      "Nearest to 重: 授, 漆, 蕾, 频, 搀, 囗, 垂, 春,\n",
      "Nearest to 更: 陛, 但, 又, 始, 窊, 且, 嗅, 枝,\n",
      "Nearest to 君: 我, 谁, 伊, 要, 选, 公, 囗, 懵,\n",
      "Nearest to ，: 、, 。, ）, 俱, 盍, 港, 圈, 即,\n",
      "Nearest to 处: 榕, 也, 迥, 著, 莘, 时, 灏, 意,\n",
      "Nearest to 寒: 秋, 暖, 枨, 暮, 鞞, 伪, 裘, 晴,\n",
      "Average loss at step  92000 :  4.14629143083\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average loss at step  94000 :  4.14613818401\n",
      "Average loss at step  96000 :  4.15640638781\n",
      "Average loss at step  98000 :  4.17982326281\n",
      "Average loss at step  100000 :  4.20927968287\n",
      "Nearest to 是: 在, 有, 道, 到, 似, 被, 把, …,\n",
      "Nearest to 三: 二, 六, 七, 五, 经, 四, 十, 维,\n",
      "Nearest to 行: 贾, 效, 暨, 飏, 岊, 送, 别, 厢,\n",
      "Nearest to 中: 里, 柝, 瀼, 答, 臧, 茯, 清, 赖,\n",
      "Nearest to 生: 怯, 缵, 丐, 筠, 乙, 冬, 予, 摧,\n",
      "Nearest to 东: 西, 北, 废, 南, 圻, 蘅, 咸, 漂,\n",
      "Nearest to 似: 如, 在, 崦, 是, 奈, 碌, 见, 论,\n",
      "Nearest to 多: 也, 巴, 甚, 镊, 浓, 缬, 奈, 却,\n",
      "Nearest to 前: 旟, 俏, 阍, 湍, 玳, 褐, 晏, 苹,\n",
      "Nearest to 深: 关, 巧, 浅, 静, 怨, 茨, 浓, 孰,\n",
      "Nearest to 重: 授, 频, 蕾, 漆, 春, 搀, 囗, 吻,\n",
      "Nearest to 更: 陛, 又, 但, 且, 窊, 始, 枝, 共,\n",
      "Nearest to 君: 我, 谁, 伊, 公, 要, 选, 他, 囗,\n",
      "Nearest to ，: 、, 。, ）, 俱, 豉, 牒, 即, 寡,\n",
      "Nearest to 处: 榕, 也, 时, 灏, 莘, 迥, 穷, 醿,\n",
      "Nearest to 寒: 秋, 暖, 暮, 枨, 鞞, 痕, 覆, 冷,\n",
      "Average loss at step  102000 :  4.20731769776\n",
      "Average loss at step  104000 :  4.22035507965\n",
      "Average loss at step  106000 :  4.1899474057\n",
      "Average loss at step  108000 :  4.20929305589\n",
      "Average loss at step  110000 :  4.245708565\n",
      "Nearest to 是: 在, 有, 似, 道, 到, 把, 被, 储,\n",
      "Nearest to 三: 七, 二, 六, 五, 经, 几, 维, 扁,\n",
      "Nearest to 行: 贾, 效, 暨, 别, 送, 岊, 厢, 飏,\n",
      "Nearest to 中: 里, 柝, 瀼, 臧, 清, 答, 洁, 茯,\n",
      "Nearest to 生: 缵, 怯, 丐, 筠, 乙, 冬, 予, 世,\n",
      "Nearest to 东: 西, 北, 废, 南, 圻, 逝, 蘅, 咸,\n",
      "Nearest to 似: 如, 在, 奈, 是, 见, 崦, 碌, 又,\n",
      "Nearest to 多: 巴, 镊, 也, 甚, 浓, 缬, 却, 搅,\n",
      "Nearest to 前: 俏, 旟, 玳, 湍, 阍, 晏, 苹, 〔,\n",
      "Nearest to 深: 关, 巧, 浅, 静, 怨, 茨, 浓, 何,\n",
      "Nearest to 重: 授, 漆, 频, 蕾, 伯, 喔, 再, 搀,\n",
      "Nearest to 更: 又, 陛, 但, 且, 窊, 共, 枝, 便,\n",
      "Nearest to 君: 我, 伊, 公, 谁, 要, 选, 沱, 他,\n",
      "Nearest to ，: 、, 。, ）, 俱, 之, ·, 郢, ,,\n",
      "Nearest to 处: 榕, 也, 迥, 莘, 灏, 时, 著, 穷,\n",
      "Nearest to 寒: 秋, 暖, 冷, 暮, 伪, 着, 鞞, 枨,\n",
      "Average loss at step  112000 :  4.18569677341\n",
      "Average loss at step  114000 :  4.10095158386\n",
      "Average loss at step  116000 :  4.11486381507\n",
      "Average loss at step  118000 :  4.10002368021\n",
      "Average loss at step  120000 :  4.10910085428\n",
      "Nearest to 是: 在, 有, 似, 到, 道, 被, 把, 过,\n",
      "Nearest to 三: 二, 五, 七, 六, 四, 经, 十, 八,\n",
      "Nearest to 行: 贾, 效, 送, 岊, 厢, 别, 暨, 过,\n",
      "Nearest to 中: 里, 瀼, 柝, 臧, 清, 茯, 答, 洁,\n",
      "Nearest to 生: 缵, 怯, 丐, 乙, 筠, 冬, 予, 摧,\n",
      "Nearest to 东: 西, 北, 南, 废, 圻, 蘅, 咸, 葵,\n",
      "Nearest to 似: 如, 在, 是, 记, 奈, 见, 崦, 甚,\n",
      "Nearest to 多: 甚, 镊, 也, 巴, 浓, 缬, 却, 犬,\n",
      "Nearest to 前: 俏, 旟, 玳, 苹, 湍, 阍, 晏, 〔,\n",
      "Nearest to 深: 关, 巧, 浅, 浓, 静, 何, 怨, 茨,\n",
      "Nearest to 重: 漆, 频, 蕾, 同, 授, 搀, 伯, 篽,\n",
      "Nearest to 更: 又, 陛, 但, 且, 始, 便, 窊, 蛉,\n",
      "Nearest to 君: 我, 谁, 伊, 要, 公, 选, 他, 沱,\n",
      "Nearest to ，: 、, 。, ）, 佣, 即, 港, 仔, 骀,\n",
      "Nearest to 处: 榕, 时, 也, 迥, 著, 捻, 莘, 灏,\n",
      "Nearest to 寒: 秋, 暖, 暮, 冷, 痕, 残, 澌, 着,\n",
      "Average loss at step  122000 :  4.12272586352\n",
      "Average loss at step  124000 :  4.11974264395\n",
      "Average loss at step  126000 :  4.14831240463\n",
      "Average loss at step  128000 :  4.17502414775\n",
      "Average loss at step  130000 :  4.17493456662\n",
      "Nearest to 是: 在, 道, 有, 把, 似, 被, 到, 过,\n",
      "Nearest to 三: 二, 六, 七, 五, 十, 经, 四, 八,\n",
      "Nearest to 行: 贾, 效, 暨, 送, 别, 岊, 厢, 贱,\n",
      "Nearest to 中: 里, 柝, 清, 瀼, 臧, 洁, 岘, 茯,\n",
      "Nearest to 生: 怯, 缵, 乙, 丐, 予, 冬, 摧, 筠,\n",
      "Nearest to 东: 西, 北, 南, 圻, 废, 蘅, 咸, 逝,\n",
      "Nearest to 似: 如, 在, 奈, 崦, 记, 是, 羡, 、,\n",
      "Nearest to 多: 巴, 镊, 也, 浓, 甚, 搅, 何, 却,\n",
      "Nearest to 前: 俏, 玳, 旟, 阍, 湍, 苹, 晏, 〔,\n",
      "Nearest to 深: 关, 巧, 浅, 怨, 浓, 静, 茨, 啼,\n",
      "Nearest to 重: 频, 授, 漆, 蕾, 再, 伯, 春, 同,\n",
      "Nearest to 更: 又, 陛, 但, 窊, 蛉, 便, 且, 共,\n",
      "Nearest to 君: 我, 伊, 公, 谁, 他, 要, 选, 沱,\n",
      "Nearest to ，: 、, 。, ）, 颔, 即, 巳, 港, 甜,\n",
      "Nearest to 处: 榕, 也, 时, 迥, 灏, 莘, 捻, 穷,\n",
      "Nearest to 寒: 秋, 暖, 冷, 阔, 暮, 着, 劈, 枨,\n",
      "Average loss at step  132000 :  4.19243708134\n",
      "Average loss at step  134000 :  4.16238326931\n",
      "Average loss at step  136000 :  4.17898646772\n",
      "Average loss at step  138000 :  4.20910188687\n",
      "Average loss at step  140000 :  4.14914165092\n",
      "Nearest to 是: 在, 似, 有, 把, 道, 被, 到, 好,\n",
      "Nearest to 三: 二, 五, 七, 六, 十, 八, 四, 鞘,\n",
      "Nearest to 行: 贾, 送, 效, 暨, 岊, 别, 嘿, 贱,\n",
      "Nearest to 中: 里, 柝, 瀼, 清, 臧, 答, 悰, 洁,\n",
      "Nearest to 生: 怯, 缵, 丐, 乙, 世, 摧, 旺, 幂,\n",
      "Nearest to 东: 西, 北, 南, 废, 圻, 逝, 蘅, 咸,\n",
      "Nearest to 似: 如, 奈, 在, 是, 记, 见, 羡, 崦,\n",
      "Nearest to 多: 浓, 甚, 巴, 也, 何, 镊, 搅, 穿,\n",
      "Nearest to 前: 俏, 玳, 旟, 湍, 苹, 晏, 阍, 〔,\n",
      "Nearest to 深: 关, 浅, 巧, 静, 怨, 浓, 密, 茨,\n",
      "Nearest to 重: 频, 授, 漆, 再, 蕾, 同, 初, 吻,\n",
      "Nearest to 更: 又, 便, 陛, 但, 且, 共, 窊, 蛉,\n",
      "Nearest to 君: 我, 伊, 公, 谁, 他, 选, 要, 沱,\n",
      "Nearest to ，: 、, 。, ）, 之, （, 牒, 港, 佣,\n",
      "Nearest to 处: 榕, 迥, 穷, 捻, 时, 也, 灏, 莘,\n",
      "Nearest to 寒: 秋, 暖, 冷, 残, 着, 暮, 墓, 痕,\n",
      "Average loss at step  142000 :  4.07607341373\n",
      "Average loss at step  144000 :  4.09916434777\n",
      "Average loss at step  146000 :  4.07604909039\n",
      "Average loss at step  148000 :  4.08231879592\n",
      "Average loss at step  150000 :  4.09918425375\n",
      "Nearest to 是: 在, 有, 似, 把, 过, 到, 被, 道,\n",
      "Nearest to 三: 二, 五, 七, 六, 八, 四, 十, 鞘,\n",
      "Nearest to 行: 贾, 效, 别, 送, 岊, 嘿, 驻, 过,\n",
      "Nearest to 中: 里, 柝, 瀼, 臧, 悰, 洁, 覆, 王,\n",
      "Nearest to 生: 缵, 怯, 乙, 丐, 冬, 世, 旺, 予,\n",
      "Nearest to 东: 西, 北, 南, 圻, 废, 蘅, 昭, 咸,\n",
      "Nearest to 似: 如, 奈, 在, 是, 记, 羡, 崦, 胜,\n",
      "Nearest to 多: 浓, 镊, 甚, 巴, 减, 也, 何, 犬,\n",
      "Nearest to 前: 俏, 玳, 旟, 湍, 苹, 阍, 〔, 晏,\n",
      "Nearest to 深: 关, 浅, 巧, 静, 浓, 怨, 密, 啼,\n",
      "Nearest to 重: 频, 蕾, 同, 漆, 授, 再, 篽, 省,\n",
      "Nearest to 更: 又, 便, 但, 陛, 且, 共, 窊, 蛉,\n",
      "Nearest to 君: 我, 伊, 谁, 公, 要, 他, 返, 懵,\n",
      "Nearest to ，: 、, 。, ）, 即, 佣, 俱, 粮, 豉,\n",
      "Nearest to 处: 榕, 捻, 时, 穷, 著, 莘, 迥, 限,\n",
      "Nearest to 寒: 秋, 暖, 着, 冷, 暮, 澌, 痕, 厉,\n",
      "Average loss at step  152000 :  4.08875273228\n",
      "Average loss at step  154000 :  4.1240436728\n",
      "Average loss at step  156000 :  4.15064696074\n",
      "Average loss at step  158000 :  4.14922552454\n",
      "Average loss at step  160000 :  4.1684006691\n",
      "Nearest to 是: 在, 把, 道, 似, 到, 有, 被, 好,\n",
      "Nearest to 三: 五, 六, 七, 二, 八, 四, 十, 几,\n",
      "Nearest to 行: 贾, 效, 别, 送, 岊, 暨, 驻, 引,\n",
      "Nearest to 中: 里, 柝, 瀼, 岘, 臧, 清, 洁, 乾,\n",
      "Nearest to 生: 怯, 缵, 乙, 丐, 予, 冬, 世, 嘴,\n",
      "Nearest to 东: 西, 北, 南, 圻, 咸, 废, 逝, 蘅,\n",
      "Nearest to 似: 如, 奈, 在, 记, 是, 羡, 崦, 共,\n",
      "Nearest to 多: 浓, 巴, 镊, 甚, 穿, 搅, 何, 也,\n",
      "Nearest to 前: 俏, 玳, 苹, 旟, 湍, 阍, 晏, 〔,\n",
      "Nearest to 深: 浅, 关, 巧, 浓, 怨, 静, 啼, 密,\n",
      "Nearest to 重: 频, 漆, 再, 蕾, 授, 伯, 吻, 同,\n",
      "Nearest to 更: 又, 但, 便, 陛, 蛉, 共, 窊, 枝,\n",
      "Nearest to 君: 我, 公, 伊, 谁, 他, 懵, 要, 选,\n",
      "Nearest to ，: 、, 。, ）, 即, 豉, 之, 于, 粮,\n",
      "Nearest to 处: 榕, 迥, 捻, 穷, 具, 灏, 著, 限,\n",
      "Nearest to 寒: 暖, 秋, 冷, 着, 姿, 阔, 厉, 伪,\n",
      "Average loss at step  162000 :  4.1382793541\n",
      "Average loss at step  164000 :  4.15350858176\n",
      "Average loss at step  166000 :  4.18065601099\n",
      "Average loss at step  168000 :  4.12492174804\n",
      "Average loss at step  170000 :  4.05342363989\n",
      "Nearest to 是: 在, 似, 把, 有, 被, 到, 道, 过,\n",
      "Nearest to 三: 五, 二, 七, 六, 八, 十, 四, 鞘,\n",
      "Nearest to 行: 岊, 贾, 送, 效, 驻, 嘿, 贱, 别,\n",
      "Nearest to 中: 里, 柝, 瀼, 悰, 洁, 清, 覆, 臧,\n",
      "Nearest to 生: 怯, 缵, 丐, 乙, 世, 旺, 仅, 悰,\n",
      "Nearest to 东: 西, 北, 南, 圻, 逝, 蘅, 废, 咸,\n",
      "Nearest to 似: 如, 奈, 是, 在, 记, 羡, 又, 见,\n",
      "Nearest to 多: 浓, 何, 甚, 穿, 也, 镊, 减, 巴,\n",
      "Nearest to 前: 俏, 玳, 旟, 苹, 晏, 湍, 〔, 阍,\n",
      "Nearest to 深: 浅, 关, 巧, 浓, 密, 怨, 静, 苘,\n",
      "Nearest to 重: 频, 再, 蕾, 漆, 同, 授, 初, 密,\n",
      "Nearest to 更: 便, 又, 陛, 但, 且, 共, 窊, 最,\n",
      "Nearest to 君: 我, 伊, 公, 谁, 他, 要, 懵, 返,\n",
      "Nearest to ，: 、, 。, ）, ,, 萨, 豉, 臑, 忤,\n",
      "Nearest to 处: 穷, 限, 榕, 具, 迥, 捻, 时, 著,\n",
      "Nearest to 寒: 秋, 暖, 冷, 残, 着, 痕, 墓, 冻,\n",
      "Average loss at step  172000 :  4.08430706024\n",
      "Average loss at step  174000 :  4.04532182884\n",
      "Average loss at step  176000 :  4.06253383446\n",
      "Average loss at step  178000 :  4.0894243983\n",
      "Average loss at step  180000 :  4.06977433169\n",
      "Nearest to 是: 在, 有, 把, 似, 被, 道, 过, 到,\n",
      "Nearest to 三: 五, 二, 六, 七, 八, 四, 十, 鞘,\n",
      "Nearest to 行: 贾, 效, 送, 驻, 别, 岊, 嘿, 车,\n",
      "Nearest to 中: 里, 瀼, 柝, 悰, 赖, 洁, 王, 臧,\n",
      "Nearest to 生: 怯, 缵, 丐, 乙, 旺, 予, 嘴, 仅,\n",
      "Nearest to 东: 西, 北, 南, 圻, 蘅, 废, 岘, 咸,\n",
      "Nearest to 似: 如, 奈, 记, 羡, 是, 在, 崦, 辈,\n",
      "Nearest to 多: 浓, 何, 减, 也, 闷, 镊, 巴, 收,\n",
      "Nearest to 前: 俏, 玳, 旟, 湍, 苹, 晏, 〔, 阍,\n",
      "Nearest to 深: 浅, 关, 巧, 浓, 密, 怨, 啼, 静,\n",
      "Nearest to 重: 频, 蕾, 再, 同, 授, 漆, 篽, 春,\n",
      "Nearest to 更: 便, 又, 但, 陛, 蛉, 共, 窊, 最,\n",
      "Nearest to 君: 我, 伊, 公, 谁, 他, 要, 懵, 返,\n",
      "Nearest to ，: 、, 。, ）, 佣, 即, 遄, 粮, 以,\n",
      "Nearest to 处: 榕, 捻, 著, 限, 穷, 迥, 时, 莘,\n",
      "Nearest to 寒: 秋, 暖, 着, 冷, 暮, 厉, 痕, 阔,\n",
      "Average loss at step  182000 :  4.10053037882\n",
      "Average loss at step  184000 :  4.14120025229\n",
      "Average loss at step  186000 :  4.12729708302\n",
      "Average loss at step  188000 :  4.14518422461\n",
      "Average loss at step  190000 :  4.11709366882\n",
      "Nearest to 是: 在, 把, 似, 有, 道, 被, 过, 负,\n",
      "Nearest to 三: 五, 二, 七, 六, 八, 十, 数, 四,\n",
      "Nearest to 行: 驻, 贾, 送, 别, 效, 岊, 暨, 嘿,\n",
      "Nearest to 中: 里, 柝, 岘, 洁, 瀼, 悰, 赖, 覆,\n",
      "Nearest to 生: 怯, 缵, 丐, 乙, 动, 仅, 戊, 摧,\n",
      "Nearest to 东: 西, 北, 南, 圻, 逝, 咸, 昭, 蘅,\n",
      "Nearest to 似: 如, 奈, 记, 是, 羡, 在, 又, 崦,\n",
      "Nearest to 多: 浓, 巴, 何, 镊, 也, 搅, 甚, 穿,\n",
      "Nearest to 前: 俏, 玳, 苹, 旟, 湍, 晏, 阍, 〔,\n",
      "Nearest to 深: 浅, 关, 浓, 巧, 静, 密, 啼, 怨,\n",
      "Nearest to 重: 蕾, 再, 频, 漆, 授, 伯, 同, 吻,\n",
      "Nearest to 更: 又, 便, 最, 蛉, 但, 共, 陛, 正,\n",
      "Nearest to 君: 我, 公, 伊, 谁, 他, 要, 返, 懵,\n",
      "Nearest to ，: 、, 。, ）, 佣, 猴, 牒, 豉, （,\n",
      "Nearest to 处: 捻, 榕, 穷, 迥, 限, 著, 具, 伦,\n",
      "Nearest to 寒: 秋, 暖, 冷, 着, 墓, 阔, 澌, 伪,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average loss at step  192000 :  4.13549966645\n",
      "Average loss at step  194000 :  4.16363530302\n",
      "Average loss at step  196000 :  4.09468750155\n",
      "Average loss at step  198000 :  4.02873082781\n",
      "Average loss at step  200000 :  4.08357236981\n",
      "Nearest to 是: 在, 把, 似, 有, 被, 到, 恐, 道,\n",
      "Nearest to 三: 五, 二, 七, 六, 八, 四, 十, 九,\n",
      "Nearest to 行: 岊, 驻, 嘿, 送, 贾, 车, 效, 别,\n",
      "Nearest to 中: 里, 柝, 悰, 洁, 瀼, 覆, 清, 岘,\n",
      "Nearest to 生: 怯, 缵, 动, 丐, 旺, 仅, 乙, 悰,\n",
      "Nearest to 东: 西, 北, 南, 昭, 蘅, 咸, 圻, 逝,\n",
      "Nearest to 似: 如, 奈, 是, 记, 在, 羡, 又, 共,\n",
      "Nearest to 多: 浓, 减, 闷, 搅, 何, 收, 镊, 滋,\n",
      "Nearest to 前: 俏, 玳, 苹, 旟, 湍, 〔, 晏, 阍,\n",
      "Nearest to 深: 关, 浅, 密, 静, 浓, 巧, 怨, 啼,\n",
      "Nearest to 重: 频, 蕾, 漆, 再, 密, 同, 授, 靓,\n",
      "Nearest to 更: 便, 又, 但, 陛, 蛉, 窊, 共, 最,\n",
      "Nearest to 君: 我, 伊, 公, 谁, 他, 说, 返, 要,\n",
      "Nearest to ，: 、, 。, ）, 遄, 俱, 之, 蚓, 豉,\n",
      "Nearest to 处: 穷, 捻, 限, 榕, 著, 迥, 具, 莘,\n",
      "Nearest to 寒: 秋, 暖, 着, 墓, 冷, 痕, 残, 澌,\n",
      "Average loss at step  202000 :  4.02759117389\n",
      "Average loss at step  204000 :  4.04968553448\n",
      "Average loss at step  206000 :  4.06768967336\n",
      "Average loss at step  208000 :  4.0558720299\n",
      "Average loss at step  210000 :  4.07835210896\n",
      "Nearest to 是: 在, 把, 似, 道, 过, 被, 到, 有,\n",
      "Nearest to 三: 五, 二, 六, 七, 八, 四, 十, 九,\n",
      "Nearest to 行: 贾, 送, 驻, 别, 嘿, 五, 效, 岊,\n",
      "Nearest to 中: 柝, 里, 悰, 瀼, 岘, 赖, 王, 洁,\n",
      "Nearest to 生: 怯, 缵, 乙, 冬, 戊, 旺, 嘴, 悰,\n",
      "Nearest to 东: 西, 北, 南, 圻, 昭, 逝, 蘅, 江,\n",
      "Nearest to 似: 如, 奈, 记, 羡, 是, 共, 辈, 崦,\n",
      "Nearest to 多: 浓, 何, 闷, 搅, 收, 巴, 穿, 也,\n",
      "Nearest to 前: 俏, 玳, 旟, 苹, 湍, 晏, 〔, 阍,\n",
      "Nearest to 深: 浅, 关, 密, 浓, 巧, 怨, 静, 啼,\n",
      "Nearest to 重: 频, 蕾, 再, 漆, 同, 密, 篽, 伯,\n",
      "Nearest to 更: 便, 又, 蛉, 但, 陛, 窊, 最, 共,\n",
      "Nearest to 君: 我, 伊, 公, 谁, 他, 你, 懵, 要,\n",
      "Nearest to ，: 、, 。, ）, 佣, ,, 即, 将, ,\n",
      "Nearest to 处: 捻, 榕, 穷, 限, 迥, 著, 莘, 灏,\n",
      "Nearest to 寒: 秋, 暖, 着, 冷, 畏, 澌, 厉, 阔,\n",
      "Average loss at step  212000 :  4.13200470543\n",
      "Average loss at step  214000 :  4.10835154331\n",
      "Average loss at step  216000 :  4.12855426538\n",
      "Average loss at step  218000 :  4.10146841323\n",
      "Average loss at step  220000 :  4.12302717698\n",
      "Nearest to 是: 在, 把, 似, 道, 被, 负, 恨, 值,\n",
      "Nearest to 三: 五, 二, 七, 六, 八, 四, 十, 鞘,\n",
      "Nearest to 行: 贾, 别, 驻, 效, 岊, 捱, 送, 嘿,\n",
      "Nearest to 中: 里, 柝, 悰, 瀼, 洁, 臧, 覆, 清,\n",
      "Nearest to 生: 缵, 怯, 乙, 仅, 丐, 世, 动, 旺,\n",
      "Nearest to 东: 西, 北, 南, 逝, 岘, 咸, 蘅, 圻,\n",
      "Nearest to 似: 如, 奈, 羡, 记, 是, 在, 又, 崦,\n",
      "Nearest to 多: 浓, 何, 巴, 搅, 镊, 收, 穿, 渗,\n",
      "Nearest to 前: 俏, 玳, 苹, 旟, 晏, 湍, 〔, 壁,\n",
      "Nearest to 深: 浅, 关, 浓, 密, 巧, 静, 啼, 怨,\n",
      "Nearest to 重: 蕾, 再, 频, 漆, 密, 伯, 授, 吻,\n",
      "Nearest to 更: 便, 又, 最, 但, 正, 蛉, 窊, 共,\n",
      "Nearest to 君: 我, 公, 伊, 谁, 畜, 他, 懵, 沱,\n",
      "Nearest to ，: 、, 。, ）, 之, 遄, 饕, 豉, 查,\n",
      "Nearest to 处: 捻, 榕, 迥, 穷, 限, 具, 也, 莘,\n",
      "Nearest to 寒: 秋, 暖, 冷, 着, 冻, 墓, 澌, 残,\n",
      "Average loss at step  222000 :  4.14358368552\n",
      "Average loss at step  224000 :  4.07069471371\n",
      "Average loss at step  226000 :  4.02555221355\n",
      "Average loss at step  228000 :  4.06627711666\n",
      "Average loss at step  230000 :  4.01129130435\n",
      "Nearest to 是: 在, 把, 似, 有, 好, 被, 值, 到,\n",
      "Nearest to 三: 二, 五, 七, 六, 八, 四, 十, 九,\n",
      "Nearest to 行: 驻, 嘿, 贾, 车, 送, 捱, 效, 岊,\n",
      "Nearest to 中: 里, 柝, 悰, 瀼, 清, 洁, 岘, 覆,\n",
      "Nearest to 生: 缵, 乙, 旺, 怯, 丐, 动, 嘴, 仅,\n",
      "Nearest to 东: 西, 北, 南, 蘅, 昭, 圻, 岘, 逝,\n",
      "Nearest to 似: 如, 奈, 记, 羡, 是, 共, 在, 与,\n",
      "Nearest to 多: 浓, 滋, 何, 闷, 搅, 镊, 渗, 减,\n",
      "Nearest to 前: 俏, 玳, 苹, 旟, 〔, 晏, 今, 湍,\n",
      "Nearest to 深: 浅, 浓, 关, 密, 静, 何, 巧, 啼,\n",
      "Nearest to 重: 蕾, 频, 再, 漆, 密, 同, 忽, 邴,\n",
      "Nearest to 更: 便, 又, 陛, 但, 最, 共, 窊, 蛉,\n",
      "Nearest to 君: 我, 公, 伊, 谁, 他, 酬, 说, 要,\n",
      "Nearest to ，: 、, 。, ）, 驄, 佣, 沾, 豉, 入,\n",
      "Nearest to 处: 捻, 限, 著, 穷, 榕, 具, 迥, 也,\n",
      "Nearest to 寒: 暖, 秋, 着, 澌, 冷, 冻, 墓, 厉,\n",
      "Average loss at step  232000 :  4.03980711961\n",
      "Average loss at step  234000 :  4.04871058178\n",
      "Average loss at step  236000 :  4.0405130434\n",
      "Average loss at step  238000 :  4.06796404541\n",
      "Average loss at step  240000 :  4.12957379484\n",
      "Nearest to 是: 在, 把, 道, 似, 被, 过, 有, 负,\n",
      "Nearest to 三: 五, 二, 六, 七, 八, 四, 十, 九,\n",
      "Nearest to 行: 驻, 别, 嘿, 贾, 送, 效, 捱, 车,\n",
      "Nearest to 中: 柝, 里, 悰, 岘, 洁, 瀼, 臧, 覆,\n",
      "Nearest to 生: 乙, 旺, 缵, 冬, 丐, 庚, 嘴, 仅,\n",
      "Nearest to 东: 西, 北, 南, 圻, 岘, 昭, 蘅, 咸,\n",
      "Nearest to 似: 如, 奈, 记, 羡, 是, 在, 辈, 共,\n",
      "Nearest to 多: 浓, 何, 滋, 渗, 闷, 穿, 巴, 搅,\n",
      "Nearest to 前: 俏, 玳, 旟, 苹, 今, 晏, 〔, 湍,\n",
      "Nearest to 深: 浅, 密, 浓, 啼, 关, 静, 怨, 遮,\n",
      "Nearest to 重: 频, 蕾, 再, 漆, 密, 忽, 同, 约,\n",
      "Nearest to 更: 便, 又, 蛉, 但, 陛, 窊, 最, 正,\n",
      "Nearest to 君: 我, 公, 伊, 谁, 他, 你, 畜, 说,\n",
      "Nearest to ，: 、, 。, ）, ,, 粮, 遄, 豉, 颔,\n",
      "Nearest to 处: 捻, 限, 穷, 榕, 迥, 灏, 具, 莘,\n",
      "Nearest to 寒: 暖, 秋, 着, 阔, 冷, 厉, 墓, 畏,\n",
      "Average loss at step  242000 :  4.09913749444\n",
      "Average loss at step  244000 :  4.11101686537\n",
      "Average loss at step  246000 :  4.09205693591\n",
      "Average loss at step  248000 :  4.11437560999\n",
      "Average loss at step  250000 :  4.12597097659\n",
      "Nearest to 是: 在, 似, 把, 被, 有, 道, 过, 负,\n",
      "Nearest to 三: 五, 七, 二, 六, 八, 四, 十, 挺,\n",
      "Nearest to 行: 驻, 别, 嘿, 贾, 送, 捱, 效, 五,\n",
      "Nearest to 中: 柝, 里, 悰, 瀼, 覆, 臧, 清, 岘,\n",
      "Nearest to 生: 乙, 仅, 旺, 缵, 丐, 嘴, 世, 戊,\n",
      "Nearest to 东: 西, 北, 南, 昭, 岘, 蘅, 圻, 逝,\n",
      "Nearest to 似: 如, 奈, 是, 羡, 记, 又, 与, 见,\n",
      "Nearest to 多: 浓, 何, 渗, 巴, 搅, 滋, 穿, 镊,\n",
      "Nearest to 前: 俏, 玳, 苹, 旟, 〔, 湍, 晏, 壁,\n",
      "Nearest to 深: 浅, 密, 浓, 啼, 静, 关, 巧, 何,\n",
      "Nearest to 重: 再, 蕾, 频, 密, 漆, 靓, 旬, 忽,\n",
      "Nearest to 更: 便, 又, 最, 蛉, 正, 但, 窊, 共,\n",
      "Nearest to 君: 我, 公, 伊, 谁, 畜, 他, 返, 酬,\n",
      "Nearest to ，: 、, 。, ）, 豉, 花, ,, 俱, 蚓,\n",
      "Nearest to 处: 捻, 限, 榕, 穷, 迥, 伦, 灏, 莘,\n",
      "Nearest to 寒: 暖, 秋, 着, 冷, 墓, 冻, 厉, 澌,\n",
      "Average loss at step  252000 :  4.04762936926\n",
      "Average loss at step  254000 :  4.02048133731\n",
      "Average loss at step  256000 :  4.05444761062\n",
      "Average loss at step  258000 :  4.00007075369\n",
      "Average loss at step  260000 :  4.02532434416\n",
      "Nearest to 是: 在, 似, 把, 值, 有, 被, 道, 过,\n",
      "Nearest to 三: 五, 二, 六, 七, 八, 四, 岱, 十,\n",
      "Nearest to 行: 别, 车, 嘿, 驻, 送, 贾, 捱, 岊,\n",
      "Nearest to 中: 里, 柝, 悰, 瀼, 洁, 覆, 岘, 臧,\n",
      "Nearest to 生: 缵, 旺, 乙, 戊, 嘴, 仅, 冬, 悰,\n",
      "Nearest to 东: 西, 北, 南, 蘅, 昭, 圻, 岘, 王,\n",
      "Nearest to 似: 如, 奈, 记, 是, 羡, 共, 在, 知,\n",
      "Nearest to 多: 浓, 闷, 滋, 何, 搅, 渗, 镊, 穿,\n",
      "Nearest to 前: 俏, 玳, 苹, 旟, 今, 〔, 晏, 湍,\n",
      "Nearest to 深: 浅, 密, 浓, 静, 啼, 关, 何, 怨,\n",
      "Nearest to 重: 再, 蕾, 频, 同, 密, 忽, 漆, 旬,\n",
      "Nearest to 更: 便, 又, 最, 但, 蛉, 陛, 夜, 渐,\n",
      "Nearest to 君: 我, 公, 谁, 伊, 他, 须, 说, 酬,\n",
      "Nearest to ，: 、, 。, ）, 佣, 豉, 遄, 俱, ；,\n",
      "Nearest to 处: 限, 捻, 穷, 榕, 著, 地, 更, 迥,\n",
      "Nearest to 寒: 秋, 暖, 着, 澌, 冻, 冷, 厉, 畏,\n",
      "Average loss at step  262000 :  4.03867952773\n",
      "Average loss at step  264000 :  4.02801381052\n",
      "Average loss at step  266000 :  4.0594327265\n",
      "Average loss at step  268000 :  4.11059404111\n",
      "Average loss at step  270000 :  4.08302716136\n",
      "Nearest to 是: 在, 把, 道, 似, 过, 有, 好, 值,\n",
      "Nearest to 三: 五, 二, 六, 七, 八, 十, 四, 数,\n",
      "Nearest to 行: 驻, 送, 捱, 别, 贾, 嘿, 效, 陈,\n",
      "Nearest to 中: 柝, 里, 岘, 悰, 覆, 洁, 臧, 瀼,\n",
      "Nearest to 生: 乙, 缵, 戊, 嘴, 庚, 波, 丐, 旺,\n",
      "Nearest to 东: 西, 北, 南, 圻, 蘅, 昭, 逝, 岘,\n",
      "Nearest to 似: 如, 奈, 记, 羡, 是, 知, 共, 与,\n",
      "Nearest to 多: 浓, 何, 闷, 渗, 滋, 巴, 搅, 穿,\n",
      "Nearest to 前: 俏, 玳, 苹, 旟, 〔, 晏, 今, 壁,\n",
      "Nearest to 深: 浓, 密, 浅, 静, 啼, 关, 蘸, 巧,\n",
      "Nearest to 重: 再, 频, 密, 蕾, 漆, 同, 吻, 靓,\n",
      "Nearest to 更: 便, 最, 又, 蛉, 正, 窊, 陛, 但,\n",
      "Nearest to 君: 我, 公, 伊, 谁, 他, 你, 说, 懵,\n",
      "Nearest to ，: 。, 、, ）, （, 秋, 佣, 春, 臑,\n",
      "Nearest to 处: 捻, 限, 穷, 榕, 迥, 地, 揎, 更,\n",
      "Nearest to 寒: 暖, 秋, 冷, 着, 冻, 阔, 澌, 厉,\n",
      "Average loss at step  272000 :  4.09433970416\n",
      "Average loss at step  274000 :  4.08163651228\n",
      "Average loss at step  276000 :  4.10384845102\n",
      "Average loss at step  278000 :  4.10842863512\n",
      "Average loss at step  280000 :  4.03459894657\n",
      "Nearest to 是: 在, 把, 似, 有, 被, 值, 过, 。,\n",
      "Nearest to 三: 五, 二, 七, 六, 八, 十, 四, 九,\n",
      "Nearest to 行: 驻, 车, 嘿, 送, 别, 捱, 五, 岊,\n",
      "Nearest to 中: 柝, 里, 悰, 覆, 岘, 瀼, 洁, 喷,\n",
      "Nearest to 生: 缵, 仅, 旺, 悰, 丐, □, 动, 乙,\n",
      "Nearest to 东: 西, 北, 南, 蘅, 逝, 岘, 昭, 圻,\n",
      "Nearest to 似: 如, 奈, 是, 记, 羡, 又, 与, 共,\n",
      "Nearest to 多: 浓, 何, 滋, 渗, 闷, 搅, 人, 穿,\n",
      "Nearest to 前: 俏, 玳, 今, 旟, 苹, 因, 〔, 晏,\n",
      "Nearest to 深: 浅, 密, 浓, 静, 啼, 关, 怨, 蘸,\n",
      "Nearest to 重: 再, 蕾, 频, 同, 密, □, 忽, 靓,\n",
      "Nearest to 更: 便, 最, 又, 共, 窊, 但, 正, 蛉,\n",
      "Nearest to 君: 我, 公, 伊, 谁, 说, 须, 他, 畜,\n",
      "Nearest to ，: 、, 。, ）, 花, 情, 佣, 忤, （,\n",
      "Nearest to 处: 限, 捻, 穷, 迥, 榕, □, 更, 时,\n",
      "Nearest to 寒: 秋, 暖, 冻, 冷, 着, 残, 墓, 瘦,\n",
      "Average loss at step  282000 :  4.02109546006\n",
      "Average loss at step  284000 :  4.03847065961\n",
      "Average loss at step  286000 :  3.99219684267\n",
      "Average loss at step  288000 :  4.01196609807\n",
      "Average loss at step  290000 :  4.03811029816\n",
      "Nearest to 是: 在, 把, 有, 似, 过, 值, 被, 道,\n",
      "Nearest to 三: 五, 二, 六, 八, 七, 四, 岱, 十,\n",
      "Nearest to 行: 车, 驻, 别, 送, 捱, 嘿, 五, 陈,\n",
      "Nearest to 中: 柝, 覆, 里, 悰, 瀼, 洁, 岘, 臧,\n",
      "Nearest to 生: 缵, 旺, 乙, 丐, 戊, 仅, 悰, 动,\n",
      "Nearest to 东: 西, 南, 北, 圻, 蘅, 岘, 昭, 江,\n",
      "Nearest to 似: 如, 奈, 记, 羡, 是, 知, 与, 共,\n",
      "Nearest to 多: 浓, 闷, 何, 滋, 渗, 减, 搅, 巴,\n",
      "Nearest to 前: 俏, 玳, 旟, 苹, 今, 湍, 〔, 晏,\n",
      "Nearest to 深: 密, 浅, 静, 浓, 啼, 巧, 何, 关,\n",
      "Nearest to 重: 蕾, 频, 同, 再, 忽, 密, 靓, 叠,\n",
      "Nearest to 更: 便, 最, 又, 但, 渐, 共, 蛉, 只,\n",
      "Nearest to 君: 我, 公, 伊, 谁, 他, 说, 要, 酬,\n",
      "Nearest to ，: 、, 。, ）, 佣, ；, 臑, 骀, □,\n",
      "Nearest to 处: 限, 捻, 穷, 榕, 更, 地, 伦, 著,\n",
      "Nearest to 寒: 秋, 暖, 着, 澌, 冷, 痕, 冻, 厉,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average loss at step  292000 :  4.02234349239\n",
      "Average loss at step  294000 :  4.05594879401\n",
      "Average loss at step  296000 :  4.09670471442\n",
      "Average loss at step  298000 :  4.06917697191\n",
      "Average loss at step  300000 :  4.0899113158\n",
      "Nearest to 是: 在, 把, 似, 道, 有, 值, 恐, 过,\n",
      "Nearest to 三: 五, 二, 六, 七, 八, 四, 十, 数,\n",
      "Nearest to 行: 驻, 别, 五, 捱, 车, 嘿, 效, 送,\n",
      "Nearest to 中: 柝, 岘, 里, 悰, 洁, 瀼, 祇, 臧,\n",
      "Nearest to 生: 缵, 仅, 动, 丐, 乙, 嘴, 戊, 熹,\n",
      "Nearest to 东: 西, 北, 南, 岘, 昭, 圻, 蘅, 逝,\n",
      "Nearest to 似: 如, 奈, 羡, 记, 是, 知, 共, 与,\n",
      "Nearest to 多: 浓, 滋, 巴, 何, 闷, 渗, 澹, 搅,\n",
      "Nearest to 前: 俏, 苹, 玳, 旟, 湍, 今, 壁, 〔,\n",
      "Nearest to 深: 浅, 浓, 密, 啼, 静, 蘸, 遮, 锁,\n",
      "Nearest to 重: 频, 再, 蕾, 同, 漆, 忽, 密, 吻,\n",
      "Nearest to 更: 便, 最, 又, 蛉, 正, 共, 窊, 奈,\n",
      "Nearest to 君: 我, 公, 伊, 谁, 说, 你, 酬, 畜,\n",
      "Nearest to ，: 、, 。, ）, 豉, 即, 有, “, 籋,\n",
      "Nearest to 处: 限, 捻, 穷, 迥, 榕, 揎, 地, 也,\n",
      "Nearest to 寒: 暖, 秋, 冷, 着, 冻, 怯, 澌, 墓,\n",
      "Average loss at step  302000 :  4.06280148184\n",
      "Average loss at step  304000 :  4.09612073314\n",
      "Average loss at step  306000 :  4.09737725306\n",
      "Average loss at step  308000 :  4.01324130738\n",
      "Average loss at step  310000 :  4.01034530926\n",
      "Nearest to 是: 在, 似, 把, 有, 过, 值, 被, 恐,\n",
      "Nearest to 三: 五, 二, 七, 八, 六, 四, 十, 一,\n",
      "Nearest to 行: 驻, 车, 送, 捱, 嘿, 五, 岊, 别,\n",
      "Nearest to 中: 柝, 里, 悰, 覆, 洁, 岘, 清, 瀼,\n",
      "Nearest to 生: 仅, 缵, 旺, 动, 悰, 丐, 乙, 摧,\n",
      "Nearest to 东: 西, 北, 南, 蘅, 岘, 昭, 圻, 逝,\n",
      "Nearest to 似: 如, 奈, 是, 记, 羡, 共, 在, 知,\n",
      "Nearest to 多: 浓, 滋, 何, 渗, 闷, 偏, 巴, 穿,\n",
      "Nearest to 前: 俏, 玳, 苹, 〔, 旟, 壁, 因, 晏,\n",
      "Nearest to 深: 密, 浅, 浓, 静, 啼, 关, 椒, 巧,\n",
      "Nearest to 重: 密, 再, 频, 同, 蕾, 忽, 叠, 靓,\n",
      "Nearest to 更: 便, 最, 又, 但, 共, 正, 嗅, 窊,\n",
      "Nearest to 君: 我, 公, 伊, 谁, 说, 他, 畜, 返,\n",
      "Nearest to ，: 、, 。, ）, （, □, 风, ；, 有,\n",
      "Nearest to 处: 限, 捻, 穷, 意, 迥, 榕, 著, 伦,\n",
      "Nearest to 寒: 暖, 秋, 冻, 墓, 冷, 瘦, 怯, 着,\n",
      "Average loss at step  312000 :  4.0321293385\n",
      "Average loss at step  314000 :  3.98314259231\n",
      "Average loss at step  316000 :  4.00005768776\n",
      "Average loss at step  318000 :  4.03388514125\n",
      "Average loss at step  320000 :  4.01660974431\n",
      "Nearest to 是: 在, 把, 似, 过, 有, 道, 被, 向,\n",
      "Nearest to 三: 五, 二, 八, 六, 七, 四, 岱, 十,\n",
      "Nearest to 行: 驻, 车, 别, 嘿, 捱, 五, 送, 陈,\n",
      "Nearest to 中: 悰, 柝, 覆, 洁, 岘, 里, 瀼, 祇,\n",
      "Nearest to 生: 缵, 悰, 旺, 仅, 丐, 伤, 嘴, 戊,\n",
      "Nearest to 东: 西, 北, 南, 岘, 圻, 蘅, 江, 昭,\n",
      "Nearest to 似: 如, 奈, 记, 羡, 是, 共, 与, 问,\n",
      "Nearest to 多: 浓, 滋, 何, 闷, 渗, 慳, 巴, 偏,\n",
      "Nearest to 前: 俏, 苹, 玳, 旟, 晏, 今, 壁, 〔,\n",
      "Nearest to 深: 密, 浅, 浓, 静, 啼, 蘸, 遮, 士,\n",
      "Nearest to 重: 频, 蕾, 密, 忽, 再, 同, 漆, 叠,\n",
      "Nearest to 更: 便, 又, 最, 但, 渐, 蛉, 共, 窊,\n",
      "Nearest to 君: 我, 公, 伊, 谁, 他, 你, 说, 客,\n",
      "Nearest to ，: 、, 。, ）, 皈, 甜, 花, ,, 有,\n",
      "Nearest to 处: 捻, 限, 穷, 榕, 地, 便, 时, 迥,\n",
      "Nearest to 寒: 秋, 暖, 着, 瘦, 冷, 冻, 振, 阒,\n",
      "Average loss at step  322000 :  4.05290821612\n",
      "Average loss at step  324000 :  4.0812803067\n",
      "Average loss at step  326000 :  4.05955528259\n",
      "Average loss at step  328000 :  4.076465011\n",
      "Average loss at step  330000 :  4.05770430398\n",
      "Nearest to 是: 在, 把, 似, 有, 值, 恨, 道, 负,\n",
      "Nearest to 三: 五, 二, 七, 八, 六, 十, 四, 两,\n",
      "Nearest to 行: 驻, 车, 捱, 别, 嘿, 五, 陈, 送,\n",
      "Nearest to 中: 柝, 里, 悰, 覆, 岘, 洁, 祇, 喷,\n",
      "Nearest to 生: 缵, 动, 仅, 乙, 丐, 戊, 悰, 嘴,\n",
      "Nearest to 东: 西, 北, 岘, 南, 昭, 江, 蘅, 圻,\n",
      "Nearest to 似: 如, 奈, 记, 是, 羡, 知, 胜, 共,\n",
      "Nearest to 多: 浓, 滋, 何, 巴, 渗, 偏, 闷, 澹,\n",
      "Nearest to 前: 俏, 苹, 玳, 旟, 今, 湍, 晏, 〔,\n",
      "Nearest to 深: 浅, 密, 浓, 静, 啼, 锁, 悄, 蘸,\n",
      "Nearest to 重: 蕾, 再, 密, 频, 忽, 漆, 靓, 叠,\n",
      "Nearest to 更: 便, 最, 又, 正, 渐, 共, 蛉, 奈,\n",
      "Nearest to 君: 我, 公, 伊, 说, 你, 畜, 谁, 予,\n",
      "Nearest to ，: 、, 。, ）, 豉, 遄, 佣, 骀, （,\n",
      "Nearest to 处: 限, 捻, 穷, 迥, 揎, 榕, 伦, 更,\n",
      "Nearest to 寒: 暖, 秋, 冷, 着, 冻, 振, 墓, 澌,\n",
      "Average loss at step  332000 :  4.08703299737\n",
      "Average loss at step  334000 :  4.09681965852\n",
      "Average loss at step  336000 :  3.99540962684\n",
      "Average loss at step  338000 :  3.9963696475\n",
      "Average loss at step  340000 :  4.03379566908\n",
      "Nearest to 是: 在, 把, 似, 值, 有, 过, 姻, 恐,\n",
      "Nearest to 三: 二, 五, 七, 八, 六, 四, 十, 岱,\n",
      "Nearest to 行: 车, 驻, 嘿, 捱, 别, 岊, 五, 陈,\n",
      "Nearest to 中: 柝, 悰, 里, 覆, 清, 祇, 瀼, 洁,\n",
      "Nearest to 生: 缵, 动, 旺, 悰, 仅, 乙, 丐, 闸,\n",
      "Nearest to 东: 西, 北, 南, 岘, 蘅, 昭, 圻, 王,\n",
      "Nearest to 似: 如, 奈, 记, 是, 羡, 知, 在, 傍,\n",
      "Nearest to 多: 浓, 滋, 闷, 何, 偏, 渗, 慳, 搅,\n",
      "Nearest to 前: 俏, 玳, 苹, 今, 旟, 〔, 晏, 因,\n",
      "Nearest to 深: 密, 浅, 浓, 静, 啼, 关, 椒, 蘸,\n",
      "Nearest to 重: 密, 蕾, 频, 叠, 忽, 同, 再, 靓,\n",
      "Nearest to 更: 便, 最, 又, 但, 杯, 渐, 夜, 奈,\n",
      "Nearest to 君: 我, 公, 伊, 谁, 说, 他, 你, 酬,\n",
      "Nearest to ，: 、, 。, ）, ；, 亦, 俱, 佣, 入,\n",
      "Nearest to 处: 限, 穷, 捻, 总, 地, 便, 迥, 也,\n",
      "Nearest to 寒: 暖, 秋, 冻, 着, 冷, 阒, 墓, 瘦,\n",
      "Average loss at step  342000 :  3.96992126012\n",
      "Average loss at step  344000 :  3.98543278688\n",
      "Average loss at step  346000 :  4.02865096843\n",
      "Average loss at step  348000 :  4.00916682696\n",
      "Average loss at step  350000 :  4.05181399322\n",
      "Nearest to 是: 在, 把, 似, 道, 过, 值, 有, 圆,\n",
      "Nearest to 三: 五, 二, 八, 七, 六, 四, 十, 岱,\n",
      "Nearest to 行: 车, 驻, 嘿, 五, 别, 捱, 送, 陈,\n",
      "Nearest to 中: 柝, 悰, 覆, 岘, 洁, 祇, 霙, 箴,\n",
      "Nearest to 生: 悰, 缵, 嘴, 旺, 戊, 动, 仅, 乙,\n",
      "Nearest to 东: 西, 北, 南, 昭, 圻, 岘, 蘅, 赤,\n",
      "Nearest to 似: 如, 奈, 记, 羡, 是, 共, 与, 问,\n",
      "Nearest to 多: 浓, 滋, 何, 闷, 慳, 渗, 巴, 澹,\n",
      "Nearest to 前: 俏, 今, 苹, 旟, 玳, 〔, 底, 湍,\n",
      "Nearest to 深: 密, 浅, 啼, 浓, 静, 士, 怨, 蘸,\n",
      "Nearest to 重: 频, 密, 再, 蕾, 忽, 叠, 同, 漆,\n",
      "Nearest to 更: 便, 渐, 但, 又, 蛉, 最, 奈, 窊,\n",
      "Nearest to 君: 我, 公, 伊, 谁, 他, 你, 说, 返,\n",
      "Nearest to ，: 。, 、, ）, 沾, 痕, 驄, 攧, 籋,\n",
      "Nearest to 处: 限, 穷, 捻, 更, 地, 榕, 迥, 便,\n",
      "Nearest to 寒: 暖, 秋, 着, 冷, 冻, 怯, 澌, 阒,\n",
      "Average loss at step  352000 :  4.06985870087\n",
      "Average loss at step  354000 :  4.05368520296\n",
      "Average loss at step  356000 :  4.06765056455\n",
      "Average loss at step  358000 :  4.05340650594\n",
      "Average loss at step  360000 :  4.08211913908\n",
      "Nearest to 是: 在, 把, 似, 值, 负, 姻, 恨, 过,\n",
      "Nearest to 三: 五, 二, 八, 七, 六, 十, 四, 岱,\n",
      "Nearest to 行: 驻, 别, 捱, 车, 嘿, 陈, 五, 送,\n",
      "Nearest to 中: 柝, 里, 悰, 覆, 洁, 祇, 岘, 清,\n",
      "Nearest to 生: 仅, 缵, 太, 动, 乙, 嘴, 世, 丐,\n",
      "Nearest to 东: 西, 北, 南, 岘, 昭, 蘅, 圻, 王,\n",
      "Nearest to 似: 如, 奈, 羡, 记, 是, 与, 又, 知,\n",
      "Nearest to 多: 浓, 何, 滋, 巴, 渗, 澹, 闷, 偏,\n",
      "Nearest to 前: 俏, 苹, 玳, 旟, 今, 因, 壁, 〔,\n",
      "Nearest to 深: 密, 浅, 浓, 静, 啼, 士, 锁, 倬,\n",
      "Nearest to 重: 再, 蕾, 忽, 密, 频, 靓, 旬, 又,\n",
      "Nearest to 更: 便, 又, 最, 正, 蛉, 渐, 但, 窊,\n",
      "Nearest to 君: 我, 公, 伊, 谁, 你, 予, 写, 说,\n",
      "Nearest to ，: 、, 。, ）, 之, 瘠, ；, 佣, 遄,\n",
      "Nearest to 处: 限, 迥, 捻, 穷, 伦, 地, 更, 揎,\n",
      "Nearest to 寒: 暖, 秋, 冻, 冷, 振, 怯, 着, 澌,\n",
      "Average loss at step  362000 :  4.08832681739\n",
      "Average loss at step  364000 :  3.98222569656\n",
      "Average loss at step  366000 :  3.98750651479\n",
      "Average loss at step  368000 :  4.03133975697\n",
      "Average loss at step  370000 :  3.96393203306\n",
      "Nearest to 是: 在, 把, 似, 值, 有, 过, 恁, 被,\n",
      "Nearest to 三: 五, 二, 八, 七, 六, 四, 十, 岱,\n",
      "Nearest to 行: 车, 驻, 捱, 嘿, 陈, 岊, 五, 别,\n",
      "Nearest to 中: 柝, 里, 悰, 瀼, 覆, 祇, 岘, 洁,\n",
      "Nearest to 生: 缵, 旺, 戊, 乙, 动, 太, 嘴, 刻,\n",
      "Nearest to 东: 西, 北, 南, 蘅, 昭, 岘, 圻, 赤,\n",
      "Nearest to 似: 如, 奈, 记, 羡, 是, 知, 共, 胜,\n",
      "Nearest to 多: 浓, 滋, 何, 闷, 渗, 偏, 澹, 慳,\n",
      "Nearest to 前: 俏, 玳, 今, 苹, 旟, 〔, 因, 晏,\n",
      "Nearest to 深: 密, 浅, 浓, 静, 啼, 何, 倬, 士,\n",
      "Nearest to 重: 密, 蕾, 叠, 同, 再, 频, 忽, 邴,\n",
      "Nearest to 更: 便, 最, 渐, 又, 但, 蛉, 奈, 共,\n",
      "Nearest to 君: 我, 公, 伊, 谁, 说, 酬, 畜, 你,\n",
      "Nearest to ，: 、, 。, ）, ；, 佣, 遄, 驄, 随,\n",
      "Nearest to 处: 限, 捻, 地, 便, 穷, 更, 著, 榕,\n",
      "Nearest to 寒: 暖, 秋, 冻, 澌, 着, 阒, 振, 怯,\n",
      "Average loss at step  372000 :  3.9753529669\n",
      "Average loss at step  374000 :  4.01568135643\n",
      "Average loss at step  376000 :  4.00265470088\n",
      "Average loss at step  378000 :  4.04448353338\n",
      "Average loss at step  380000 :  4.06245167029\n",
      "Nearest to 是: 在, 把, 似, 有, 道, 值, 过, 恐,\n",
      "Nearest to 三: 五, 二, 七, 八, 六, 四, 岱, 十,\n",
      "Nearest to 行: 车, 驻, 嘿, 捱, 别, 五, 藁, 陈,\n",
      "Nearest to 中: 柝, 悰, 覆, 岘, 里, 祇, 霙, 榘,\n",
      "Nearest to 生: ）, 戊, 太, 缵, 乙, 旺, 庚, 悰,\n",
      "Nearest to 东: 西, 北, 南, 岘, 王, 圻, 昭, 赤,\n",
      "Nearest to 似: 如, 奈, 羡, 记, 是, 与, 知, 问,\n",
      "Nearest to 多: 浓, 滋, 何, 渗, 闷, 巴, 慳, 澹,\n",
      "Nearest to 前: 俏, 今, 苹, 玳, 旟, 〔, 壁, 槐,\n",
      "Nearest to 深: 密, 浓, 浅, 啼, 静, 士, 蘸, 倬,\n",
      "Nearest to 重: 再, 密, 频, 蕾, 忽, 漆, 同, 旬,\n",
      "Nearest to 更: 便, 又, 渐, 最, 蛉, 奈, 但, 正,\n",
      "Nearest to 君: 我, 公, 伊, 你, 谁, 说, 予, 他,\n",
      "Nearest to ，: 。, 、, ）, （, 驄, ；, 镕, 皈,\n",
      "Nearest to 处: 限, 捻, 穷, 地, 迥, 更, 趣, 灏,\n",
      "Nearest to 寒: 暖, 秋, 冻, 振, 冷, 阒, 怯, 着,\n",
      "Average loss at step  382000 :  4.04618830562\n",
      "Average loss at step  384000 :  4.05342762816\n",
      "Average loss at step  386000 :  4.04853097236\n",
      "Average loss at step  388000 :  4.07381100976\n",
      "Average loss at step  390000 :  4.07922243035\n",
      "Nearest to 是: 在, 似, 把, 有, 值, 过, 道, 负,\n",
      "Nearest to 三: 五, 二, 八, 七, 六, 十, 挺, 岱,\n",
      "Nearest to 行: 嘿, 车, 捱, 别, 驻, 五, 送, 陈,\n",
      "Nearest to 中: 柝, 悰, 里, 覆, 岘, 瀼, 喷, 清,\n",
      "Nearest to 生: 缵, 仅, 乙, 仁, 悰, 祜, 旺, 动,\n",
      "Nearest to 东: 西, 北, 南, 岘, 王, 蘅, 昭, 圻,\n",
      "Nearest to 似: 如, 奈, 是, 羡, 记, 与, 又, 共,\n",
      "Nearest to 多: 何, 滋, 浓, 巴, 渗, 闷, 澹, 偏,\n",
      "Nearest to 前: 俏, 玳, 旟, 今, 苹, 〔, 壁, 因,\n",
      "Nearest to 深: 密, 浓, 浅, 啼, 静, 蘸, 士, 锁,\n",
      "Nearest to 重: 再, 密, 蕾, 频, 忽, 同, 叠, 靓,\n",
      "Nearest to 更: 便, 最, 又, 但, 蛉, 渐, 正, 奈,\n",
      "Nearest to 君: 我, 公, 伊, 你, 予, 谁, 畜, 说,\n",
      "Nearest to ，: 、, 。, ）, 豉, ；, 遄, 瘠, 有,\n",
      "Nearest to 处: 限, 迥, 捻, 更, 趣, 穷, 榕, 地,\n",
      "Nearest to 寒: 暖, 秋, 冻, 冷, 怯, 墓, 阒, 澌,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average loss at step  392000 :  3.96885286486\n",
      "Average loss at step  394000 :  3.98929434669\n",
      "Average loss at step  396000 :  4.01414554822\n",
      "Average loss at step  398000 :  3.96302485704\n",
      "Average loss at step  400000 :  3.97452242398\n",
      "Nearest to 是: 在, 把, 似, 有, 值, 过, 道, 恁,\n",
      "Nearest to 三: 二, 五, 八, 七, 六, 十, 四, 岱,\n",
      "Nearest to 行: 车, 嘿, 别, 捱, 驻, 五, 送, 陈,\n",
      "Nearest to 中: 柝, 悰, 覆, 祇, 里, 瀼, 洁, 鹣,\n",
      "Nearest to 生: 缵, 动, 戊, 太, 旺, □, 刻, 邵,\n",
      "Nearest to 东: 西, 南, 北, 岘, 圻, 蘅, 王, 昭,\n",
      "Nearest to 似: 如, 奈, 是, 羡, 记, 知, 与, 胜,\n",
      "Nearest to 多: 浓, 滋, 何, 闷, 渗, 偏, 逸, 慳,\n",
      "Nearest to 前: 俏, 苹, 今, 旟, 玳, 先, 〔, 鹣,\n",
      "Nearest to 深: 密, 浅, 浓, 静, 啼, 椒, 何, 倬,\n",
      "Nearest to 重: 同, 密, 再, 频, 忽, 叠, 蕾, 邴,\n",
      "Nearest to 更: 便, 渐, 最, 又, 但, 奈, 蛉, 些,\n",
      "Nearest to 君: 我, 公, 伊, 谁, 说, 且, 你, 他,\n",
      "Nearest to ，: 、, 。, ）, 豉, ；, □, 俱, 趾,\n",
      "Nearest to 处: 限, 趣, 便, 穷, 更, 捻, □, 地,\n",
      "Nearest to 寒: 暖, 秋, 冻, 怯, 着, 澌, 振, 冷,\n"
     ]
    }
   ],
   "source": [
    "# Step 5: Begin training.\n",
    "num_steps = 400001\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "  # We must initialize all variables before we use them.\n",
    "  init.run()\n",
    "  print('Initialized')\n",
    "\n",
    "  average_loss = 0\n",
    "  for step in xrange(num_steps):\n",
    "    batch_inputs, batch_labels = generate_batch(\n",
    "        batch_size, num_skips, skip_window)\n",
    "    feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}\n",
    "\n",
    "    # We perform one update step by evaluating the optimizer op (including it\n",
    "    # in the list of returned values for session.run()\n",
    "    _, loss_val = session.run([optimizer, loss], feed_dict=feed_dict)\n",
    "    average_loss += loss_val\n",
    "\n",
    "    if step % 2000 == 0:\n",
    "      if step > 0:\n",
    "        average_loss /= 2000\n",
    "      # The average loss is an estimate of the loss over the last 2000 batches.\n",
    "      print('Average loss at step ', step, ': ', average_loss)\n",
    "      average_loss = 0\n",
    "\n",
    "    # Note that this is expensive (~20% slowdown if computed every 500 steps)\n",
    "    if step % 10000 == 0:\n",
    "      sim = similarity.eval()\n",
    "      for i in xrange(valid_size):\n",
    "        valid_word = reverse_dictionary[valid_examples[i]]\n",
    "        top_k = 8  # number of nearest neighbors\n",
    "        nearest = (-sim[i, :]).argsort()[1:top_k + 1]\n",
    "        log_str = 'Nearest to %s:' % valid_word\n",
    "        for k in xrange(top_k):\n",
    "          close_word = reverse_dictionary[nearest[k]]\n",
    "          log_str = '%s %s,' % (log_str, close_word)\n",
    "        print(log_str)\n",
    "  final_embeddings = normalized_embeddings.eval()\n",
    "  np.save('embedding.npy', final_embeddings)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Step 6: Visualize the embeddings.\n",
    "\n",
    "\n",
    "# pylint: disable=missing-docstring\n",
    "# Function to draw visualization of distance between embeddings.\n",
    "def plot_with_labels(low_dim_embs, labels, filename):\n",
    "  assert low_dim_embs.shape[0] >= len(labels), 'More labels than embeddings'\n",
    "  plt.figure(figsize=(18, 18))  # in inches\n",
    "  for i, label in enumerate(labels):\n",
    "    x, y = low_dim_embs[i, :]\n",
    "    plt.scatter(x, y)\n",
    "    plt.annotate(label,\n",
    "                 xy=(x, y),\n",
    "                 xytext=(5, 2),\n",
    "                 textcoords='offset points',\n",
    "                 ha='right',\n",
    "                 va='bottom')\n",
    "\n",
    "  plt.savefig(filename)\n",
    "\n",
    "try:\n",
    "  # pylint: disable=g-import-not-at-top\n",
    "  from sklearn.manifold import TSNE\n",
    "  import matplotlib.pyplot as plt\n",
    "  plt.rcParams['font.sans-serif']=['SimHei'] #用来正常显示中文标签\n",
    "  plt.rcParams['axes.unicode_minus']=False #用来正常显示负号\n",
    "\n",
    "  tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000, method='exact')\n",
    "  plot_only = 500\n",
    "  low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :])\n",
    "  labels = [reverse_dictionary[i] for i in xrange(plot_only)]\n",
    "  plot_with_labels(low_dim_embs, labels, './tsne.png')\n",
    "\n",
    "except ImportError as ex:\n",
    "  print('Please install sklearn, matplotlib, and scipy to show embeddings.')\n",
    "  print(ex)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
